# encoding: utf-8
"""
@author: youfeng
@file: spider_scheduler.py
@time: 2018/8/27 下午10:56
"""
import importlib
import os
import signal
import subprocess
import sys
import time
from threading import Thread

from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.schedulers.background import BackgroundScheduler as Scheduler

from common.runtime import RunTime
from logger import AppLogger

g_logger = AppLogger('spider_scheduler.log').get_logger()

# 当前进程运行状态
is_running = True

"""
__tasks = {
    task_path:{
        "cron" : 调度周期,
        "job" : 调度器job实例,
        "last_check" : 上次检查时间,
        "path": 代码路径/task_path
        "name": name
        "last_run": 上次运行时间
    }
}
"""


class GScheduler(object):
    def __init__(self, base_path, task_prefix="sites", sch_file_name="sch.py", logger=None):
        self.logger = logger
        self.__base_path = base_path
        self.__task_prefix = task_prefix
        self.__sch_file_name = sch_file_name
        self.__sch_module = self.__sch_file_name.split(".")[0]
        self.__tasks_dir = os.path.join(base_path, self.__task_prefix)
        self.__tasks = {
        }
        executors = {
            'default': ThreadPoolExecutor(100),
        }
        self.__scheduler = Scheduler(logger=self.logger, executors=executors, timezone="Asia/Shanghai")
        self.__scan_thread = Thread(target=self.__scan_task_config)
        self.__scan_thread.daemon = True
        self.__is_running = True
        self.__run_time = RunTime(logger)

    def process_quit(self, signo, frame):
        self.__is_running = False
        self.logger.info("收到退出进程信号...")

    @staticmethod
    def __parser_cron(string):
        minute, hour, day, month, weekday = string.split()
        cron = {
            "minute": minute,
            "hour": hour,
            "day": day,
            "month": month,
            "day_of_week": weekday
        }
        return cron

    def __do_task(self, task_path):
        try:
            task = self.__tasks[task_path]
            for cmd in task['cmd']:
                try:
                    run_cmd = "cd %s; %s" % (task['path'], cmd)
                    self.logger.info("Task start:{}\n{}".format(task['path'], run_cmd))
                    subp = subprocess.Popen(run_cmd,
                                            shell=True,
                                            stdin=open(os.devnull, "w"),
                                            stdout=open(os.devnull, "w"),
                                            stderr=sys.stderr,
                                            )
                    ret = subp.communicate()
                    if subp.returncode != 0:
                        self.logger.warning("Task return error:{}\t{}\t{}".format(task['path'], cmd, ret))
                except Exception as e:
                    self.logger.error("Task error:{}".format(e.message))
                    self.logger.exception(e)
            self.logger.info("Task finish:{}".format(task['path']))
        except Exception as e:
            self.logger.error("Execute error:{}".format(task_path))
            self.logger.exception(e)

    def __path2module(self, task_path):
        if self.__task_prefix not in task_path:
            self.logger.error("当前路径异常,不包含 {} : task_path = {}".format(self.__task_prefix, task_path))
            return None

        # task_module = task_path.split('/')[-1]
        # topic_module = task_path.split("/")[-2]

        folder_list = task_path.split(self.__task_prefix + "/")[-1].split("/")
        folder_list.append(self.__sch_module)
        folder_list.insert(0, self.__task_prefix)
        mof = ".".join(folder_list)
        if sys.modules.has_key(mof):
            reload(sys.modules[mof])
        m = importlib.import_module(mof)
        return m

    @staticmethod
    def __check_sch_config(mod):
        if not hasattr(mod, "status") or not mod.status:
            return False
        if hasattr(mod, "cron") and hasattr(mod, "site") and hasattr(mod, "cmd"):
            if not isinstance(mod.cmd, (list, tuple)) or not mod.cmd:
                return False
            if not isinstance(mod.cron, basestring) or not mod.cron:
                return False
            if not isinstance(mod.site, basestring) or not mod.site:
                return False
        else:
            return False
        return True

    def __add_task(self, task_path):
        ret = False
        try:
            m = self.__path2module(task_path)
            if self.__check_sch_config(m):
                self.logger.info("Add task:{}".format(task_path))
                task = {'site': m.site,
                        'cron': m.cron,
                        'path': task_path,
                        'last_check': -1}
                job = self.__scheduler.add_job(func=self.__do_task,
                                               args=[task_path, ],
                                               trigger="cron",
                                               **self.__parser_cron(m.cron))
                task['job'] = job
                task['cmd'] = m.cmd
                self.__tasks[task_path] = task
                self.logger.info("Add task finished:{}\t{}".format(task_path, m.cron))
                ret = True
            else:
                # self.logger.warning("Add task failed by INVALID `sch.py` FILE:{}".format(task_path))
                ret = False
        except Exception as e:
            self.logger.warning("Add task failed: {}".format(task_path))
            self.logger.exception(e)
        return ret

    def __modify_task(self, task_path):
        ret = False
        try:
            m = self.__path2module(task_path)
            if not self.__check_sch_config(m):
                self.logger.warning("Exists task has INVALID `sch.py`, will be removed:{}".format(task_path))
                self.__remove_task(task_path, kill=True)
            else:
                task = self.__tasks[task_path]
                task['site'] = m.site
                task['cron'] = m.cron
                task['path'] = task_path
                task['last_check'] = -1
                task['cmd'] = m.cmd
                self.__scheduler.remove_job(task['job'].id)
                job = self.__scheduler.add_job(func=self.__do_task,
                                               args=[task_path, ],
                                               trigger="cron",
                                               **self.__parser_cron(m.cron))
                task['job'] = job
                self.__tasks[task_path] = task
                ret = True
        except Exception as e:
            self.logger.error("What happen when modify:{}\t{}".format(task_path, e.message))
            self.logger.exception(e)
            self.__remove_task(task_path)
        return ret

    def __remove_task(self, task_path, kill=True):
        self.logger.info("Reomve task:{}".format(task_path))
        task = self.__tasks[task_path]
        self.__scheduler.remove_job(task['job'].id)
        if kill and task.has_key("process"):
            try:
                task['process'].kill()
            except Exception as e:
                self.logger.error("What happen when reomve:{}\t[}".format(task_path, e.message))
                self.logger.exception(e)
        try:
            self.__tasks.pop(task_path)
        except Exception as e:
            self.logger.error("What happen when reomve:{}\t[}".format(task_path, e.message))
            self.logger.exception(e)
        self.logger.info("Remove task:{} finished".format(task_path))

    # 添加任务路径
    def __add_task_path(self, task_path, now_time):

        conf_path = os.path.join(task_path, self.__sch_file_name)
        # 查找调度配置文件
        if os.path.isfile(conf_path):
            # 调度器中已经存在该任务路径
            if self.__tasks.has_key(task_path):
                # 判断是否有更改
                if os.path.getmtime(conf_path) > self.__tasks[task_path]['last_check']:
                    self.logger.info("Change task:{}".format(task_path))
                    # 已经存在的任务
                    if self.__modify_task(task_path):
                        self.__tasks[task_path]['last_check'] = now_time
                else:
                    self.__tasks[task_path]['last_check'] = now_time
            else:
                if self.__add_task(task_path):
                    self.logger.info("Find new task:{}".format(task_path))
                    self.__tasks[task_path]['last_check'] = now_time

    def __walk_file_path(self, parent_task_path, now_time):
        for line in os.listdir(parent_task_path):
            task_path = os.path.join(parent_task_path, line)

            # 非文件夹跳过
            if not os.path.isdir(task_path):
                continue

            self.__add_task_path(task_path, now_time)

            # 递归遍历
            self.__walk_file_path(task_path, now_time)

    def __scan_task_config(self):
        while self.__is_running:
            now_time = time.time()
            # 遍历任务目录
            for parent_line in os.listdir(self.__tasks_dir):
                parent_task_path = os.path.join(self.__tasks_dir, parent_line)
                # 非文件夹跳过
                if not os.path.isdir(parent_task_path):
                    continue

                # 遍历目录
                self.__walk_file_path(parent_task_path, now_time)

            for key, value in self.__tasks.items():
                if value['last_check'] != now_time:
                    self.logger.info("Lost task conf:{}".format(key))
                    self.__remove_task(key)
            time.sleep(10)

    def start(self):
        # 先保留PID
        self.__scan_thread.start()
        self.__scheduler.start()
        while self.__is_running:
            time.sleep(3)

        # 关闭调度器
        self.__scheduler.shutdown()
        self.logger.info("关闭调度器完成，退出")


def main():
    base_path = os.path.dirname(os.path.abspath(__file__))
    master = GScheduler(base_path, logger=g_logger)

    # 注册优雅退出
    signal.signal(signal.SIGINT, master.process_quit)
    signal.signal(signal.SIGTERM, master.process_quit)
    signal.signal(signal.SIGQUIT, master.process_quit)
    signal.signal(signal.SIGUSR1, master.process_quit)

    master.start()


if __name__ == "__main__":
    main()
